package main

import (
	"github.com/gocolly/colly"
	"fmt"
	"strings"
	"os"
	"net/http"
	"io/ioutil"
	"io"
	"bytes"
)

func main() {

	location := "nvyoubuluo\\"
	articles := location + "articles\\"


	fixPath := "https://www.nvyoubuluo.com/wp-content/themes/lensnews/includes/timthumb.php?src="

	//downloadImg(articles,"https://www.nvyoubuluo.com/wp-content/uploads/2018/01/2018010803191561.jpg")
	//downloadImg(articles,"https://f12.baidu.com/it/u=1231887722,1565021229&fm=76")
	//
	//os.Exit(0)

	index := "https://www.nvyoubuluo.com/"

	// 创建一个页面收集器
	c := colly.NewCollector(
		colly.AllowedDomains("www.nvyoubuluo.com"),
		colly.MaxDepth(10),
		//colly.Async(true),
	)

	c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 2})

	// 创建页面白名单，当页面访问过时，将页面加入该白名单
	whiteList := make([]string, 0, 500)

	// 当检测到文章标签时，点击链接进入
	c.OnHTML("article", func(element *colly.HTMLElement) {


		// 查找 figure 标签，如果找到了，说明是列表，否则遍历图片
		//figure := false
		//element.ForEach("figure", func(i int, e *colly.HTMLElement) {
		//	figure = true
		//	return
		//})

		//fmt.Printf("figure: %t\n", figure)

		// 获取标题和链接
		href := element.ChildAttr("a", "href")
		title := element.ChildAttr("a", "title")

		// 如果不是文章摘要
		if title != "" {

			dir := articles + title

			// 遍历文章下的所有图片
			element.ForEach("img", func(i int, e *colly.HTMLElement) {
				// 获取图片名称
				name := e.Attr("alt")
				// 获取图片地址
				imgPath := e.Attr("data-original")
				imgPath = delImgSize(clearFix(imgPath, fixPath))
				// 名称替换
				suf := strings.LastIndex(imgPath, ".")
				if suf != -1 {
					name = name + imgPath[suf:]
				}

				if name == "" {
					name = imgPath
				}
				downloadImg(dir, imgPath, name)
			})
		}

		// 如果超链接已经访问过，不继续操作
		for _, lis := range whiteList {
			if lis == href {
				//fmt.Printf("检测到重复的uri： %s， 已经跳过\n", href)
				return
			}
		}

		// 将已经处理过的页面加入白名单中
		whiteList = append(whiteList, href)
		fmt.Printf("whiteList length: %d\n\n", len(whiteList))
		element.Request.Visit(href)
	})

	// 当检测到img，打印之
	//c.OnHTML("img", func(element *colly.HTMLElement) {
	//	fmt.Printf("检测到img， href： %s, alt: %s\n", element.Attr("src"), element.Attr("alt"))
	//})

	// 打印请求日志
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL)
	})

	// 开始访问页面
	c.Visit(index)
}

var baseLocation = "K:\\go-download-colloy\\"

func getFolderNameFromElementChildren(e *colly.HTMLElement) string {
	fmt.Sprintf("element name: %s, text: %s", e.Name, e.Text)
	//if e.Name == "a" {
	//	if e.Text == "" {
	//		alt := e.ChildAttr("img", "alt")
	//		if alt != "" {
	//			fmt.Println(alt)
	//			return alt
	//		}
	//
	//		aclass := e.ChildAttr("section", "class")
	//		if aclass == "home_title" {
	//
	//		}
	//
	//	} else {
	//		return e.Text
	//	}
	//}
	return e.Text
}

func isImgPath(href string) bool {
	if href == "" {
		return false
	}
	i := strings.LastIndex(href, ".")
	if i == -1 {
		return false
	}
	suf := href[i+1:]
	return suf == "jpg" || suf == "jpeg" || suf == "gif" || suf == "png"
}

func downloadImg(location, href, name string) {
	if href=="" || location == "" {
		return
	}
	location = baseLocation + location

	createImg(location, href, name)
}

func clearFix(href, fix string) string {
	i := strings.Index(href, fix)
	if i == 0 {
		href = href[len(fix):]
	}
	return href
}

func delImgSize(href string) string {
	// 检测是否有尺寸标识符号
	i := strings.LastIndex(href, "&")
	if i != -1 {
		href = href[0:i]
		return delImgSize(href)
	} else {
		return href
	}
}

func createImg(local, href, name string) {

	// 判断文件是否已经下载过
	fileName := local + "\\" + name
	_, err := os.Stat(fileName)
	if err == nil {
		return
	}

	fmt.Printf("ready to download img: %s, name: %s, to dir: %s\n", href, name, local)

	// 创建文件夹
	pathExistsToCreate(local)
	// 切换文件夹到指定目录
	os.Chdir(local)

	// 创建输出文件
	out, _ := os.Create(name)
	defer out.Close()

	resp, _ := http.Get(href)
	defer resp.Body.Close()
	// 读取http响应结果
	pix, _ := ioutil.ReadAll(resp.Body)
	// 将结果写入文件流
	io.Copy(out, bytes.NewReader(pix))
}

func pathExistsToCreate(path string) {
	_, err := os.Stat(path)
	if err == nil {
		return
	}
	if os.IsNotExist(err) {
		os.MkdirAll(path, os.ModeDir)
	}
}